#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/9/20 16:32
# @Author  : 王凯
# @File    : app_crawler.py
# @Project : scrapy_spider
"https://github.com/k713927/phone_spider/blob/master/utils/PddAppRiskUtil.py"

import datetime
import hashlib
import json
import math
import random
import string
import subprocess
import time
from functools import partial

import parsel
import requests
from loguru import logger
from retrying import retry

subprocess.Popen = partial(subprocess.Popen, encoding="utf-8")  # 消除window下subprocess中文乱码问题

from apps.pdd.pdd.sync.sync_pdd_source import SyncPddSource
from apps.pdd.pdd.utils.pdd_verify import PddSliderVerify
from apps.pdd.pdd.utils.app.core.pdd_user import PDDAppUserPool
from apps.pdd.pdd.utils.app.core.app_verify import PddAppVerify
from apps.pdd.pdd.utils.js import MSG_MAP, gen_anti_content
from components.config import WFQ_SOURCE_MYSQL_CONFIG
from utils.db.mysqldb import MysqlDB
from utils.db.redisdb import RedisDB


class VerifyError(Exception):
    pass


class AppCrawler:
    _to_db = None
    _redis_db = None

    @property
    def to_db(self):
        if not self._to_db:
            self._to_db = MysqlDB(
                ip=WFQ_SOURCE_MYSQL_CONFIG["MYSQL_IP"],
                port=WFQ_SOURCE_MYSQL_CONFIG["MYSQL_PORT"],
                db=WFQ_SOURCE_MYSQL_CONFIG["MYSQL_DB"],
                user_name=WFQ_SOURCE_MYSQL_CONFIG["MYSQL_USER_NAME"],
                user_pass=WFQ_SOURCE_MYSQL_CONFIG["MYSQL_USER_PASS"],
            )
        return self._to_db

    @property
    def redis_db(self):
        if not self._redis_db:
            self._redis_db = RedisDB()
        return self._redis_db

    session = requests.Session()
    sn = {
        "mall": "10039",
        "goods_detail": "10014",
        "order_checkout": "10004",
        "personal": "10001",
        "my_order": "10032",
        "comments": "10022",
        "default": "10058",
    }

    def __init__(self, cookie, headers, proxies=None, cache_minutes=60):
        self.base_url = "https://api.pinduoduo.com"
        self.session.headers = headers
        self.session.cookies.update(cookie)
        if proxies:
            self.session.proxies = proxies
        self.pdd_user_id = cookie["pdd_user_id"]
        self.cache_minutes = cache_minutes

    def gen_anti_content(self, timestamp=int(time.time() * 1000), device_id=None):
        url = "https://sekiro.wkaanig.cn/business/invoke"
        params = {
            "group": "AntiToken",
            "sekiro_token": "c232bebc-68a8-4c3d-b310-e0d26876e58b",
            "action": "AntiTokenHandler",
            "ts": timestamp,
        }
        if device_id:
            params["device_id"] = device_id

        response = requests.get(url, params=params).json()
        logger.debug(f"获取 anti_count : {response}")
        rt = response["data"]
        return rt

    @staticmethod
    def _resolve_raw_data(html, url):
        try:
            response = parsel.Selector(html)
            raw_data = response.re_first(r"window\.rawData\s*=\s*(\{.*\});")
            if not raw_data:
                logger.error(f"未获取到数据 {url}")
                return False
        except Exception as e:
            logger.error(f"解析数据失败: {e} {url}")
            return False
        raw_data = json.loads(raw_data)
        return raw_data

    def get_mall_page_info(self, mall_id="409099714", msn=""):
        cache_time = (datetime.datetime.now() - datetime.timedelta(minutes=self.cache_minutes)).strftime(
            "%Y-%m-%d %H:%M:%S"
        )
        sql = f"select * from net_pdd_proxy_log where source = 'app' and mall_id = '{mall_id}' and create_time > '{cache_time}' and base_url = 'https://m.pinduoduo.net/mall_subject.html'"
        cache = self.to_db.find(sql, to_json=True)
        if cache:
            for i in cache:
                response = i["response"]
                if response.get("data", {}).get("store", {}).get("mallDetailInfo"):
                    logger.warning(f"获取店铺信息: 使用缓存 {mall_id} {response}")
                    return response

        url = "https://m.pinduoduo.net/mall_subject.html"
        params = {
            "_t_rpn": "comm_mall_guarantees",
            "source": "2",
            "mall_id": mall_id,
            "msn": msn,
            "page_id": f"10039_{int(time.time() * 1000)}_0{''.join(random.choices(string.digits, k=9))}",
        }
        real_url = url + "?" + "&".join([f"{k}={v}" for k, v in params.items()])
        logger.debug(f"获取店铺 url：{real_url}")
        resp = self.session.get(url, params=params).text
        raw_data = self._resolve_raw_data(resp, "mall_id")
        response = resp if resp.startswith("{") else json.dumps({"data": raw_data, "html": resp}, ensure_ascii=False)
        logger.info(f"获取店铺信息：{response}")
        save_item = {
            "request_url": real_url,
            "base_url": "https://m.pinduoduo.net/mall_subject.html",
            "mall_id": mall_id,
            "page_no": None,
            "response": response,
            "source": "app",
            "headers": self.session.headers,
        }
        self.to_db.add_smart("net_pdd_proxy_log", save_item)
        time.sleep(random.randint(5, 10))

    @retry(
        stop_max_attempt_number=3,
        wait_fixed=1000,
        retry_on_exception=lambda e: isinstance(e, VerifyError),
    )
    def get_mall_goods_info(self, mall_id, msn, page_no=1):
        cache_time = (datetime.datetime.now() - datetime.timedelta(minutes=self.cache_minutes)).strftime(
            "%Y-%m-%d %H:%M:%S"
        )
        sql = f"select * from net_pdd_proxy_log where source = 'app' and mall_id = '{mall_id}' and page_no = '{page_no}' and create_time > '{cache_time}' and base_url = 'https://api.pinduoduo.com/api/turing/mall/query_cat_goods'"
        cache = self.to_db.find(sql, to_json=True)
        if cache:
            for i in cache:
                response = i["response"]
                if response and "error_code" not in response:
                    logger.warning(f"获取到商品列表为: 使用缓存 {mall_id} page_no {page_no} {response}")
                    return response

        url = "https://api.pinduoduo.com/api/turing/mall/query_cat_goods"
        params = {
            "list_id": f"{hashlib.md5(f'{mall_id}'.encode()).hexdigest()[:10]}{mall_id}",
            "query": "",
            "refer_page_sn": "10015",
            "first_request": "1",
            "goods_id": "",
            "display_flag": "activity_region",
            "page_no": page_no,
            "type": "0",
            "show_priority_type": "0",
            "query_shipping_option": "1",
            "sort_type": "_sales",
            "page_id": "mall_page.html",
            "page_from": "39",
            "refer_pdd_type": "0",
            "category_id": "0",
            "x_query": "",
            "mall_id": mall_id,
            "goods_list_show_type": "0",
            "engine_version": "2.0",
            "page_size": "20",
            "pdduid": self.pdd_user_id,
        }
        if msn:
            params.update({"msn": msn})

        headers = {
            **self.session.headers,
            "anti-token": self.gen_anti_content(),
            #     "lat": "UJL7KFZKWE6RLFW72EHJQ5VCOR7JL3CKDEKEOZ57JRSE4EO2XRFA123a134",
            #     "p-proc-time": "580877",
            #     "x-b3-ptracer": "hctruef87b616cb33047a6bbbe11899f",
            #     "x-yak-llt": "1738389190864"
        }
        real_url = url + "?" + "&".join([f"{k}={v}" for k, v in params.items()])
        resp = self.session.get(url, params=params, headers=headers).json()
        logger.info(f"获取到商品列表为：{resp}")

        response = json.dumps(resp, ensure_ascii=False)
        save_item = {
            "request_url": real_url,
            "base_url": "https://api.pinduoduo.com/api/turing/mall/query_cat_goods",
            "mall_id": mall_id,
            "page_no": page_no,
            "response": response,
            "source": "app",
            "headers": self.session.headers,
        }
        self.to_db.add_smart("net_pdd_proxy_log", save_item)

        if resp.get("error_code"):
            if "verify_auth_token" in resp:
                verify = PddAppVerify(resp.get("verify_auth_token")).run()
                if verify:
                    self.session.headers.update({"VerifyAuthToken": resp.get("verify_auth_token")})
                    raise VerifyError(f"{resp.get('error_code')} {resp.get('error_message')}")
            raise Exception(MSG_MAP.get(resp.get("error_code")))

        if resp == {}:
            raise Exception("商品列表为空")

        time.sleep(random.randint(10, 30))
        return resp

    @retry(
        stop_max_attempt_number=3,
        wait_fixed=1000,
        retry_on_exception=lambda e: isinstance(e, VerifyError),
    )
    def get_mall_total(self, mall_id):
        """
        {"mall_id":2044177,"mall_name":"福寿臻选良品","logo":"http://t16img.yangkeduo.com/pdd_ims/484547576335d73b5ff3885a9ba6b048.jpg","goods_num":1,"mall_desc":"生活从来都只是用来享受的。","company_phone":"","offline_note":"","chat_enable":0,"refund_address":"","score_avg":0,"staple_id":11,"mall_sales":100000,"sales_tip":"已拼100万+件","goods_num_desc":"商品数量: 1","region_emergent":0,"is_open":1,"status":2,"wms_id":0,"mall_coupons":[],"is_favorite":true,"server_time":1735551274,"share_coupon_info":{"mall_base_coupon_type":0,"in_activity":false,"is_received":false},"if_brand_story":false,"super_mall_coupons":[],"mall_activity_count_down_time":30,"mall_label_list":[{"type":1,"desc":"全场包邮","background_color":"#15E02E24","script_color":"#FFE02E24","transparent_degree":"0.08"}],"immersion_flag":false}
        """
        cache_time = (datetime.datetime.now() - datetime.timedelta(minutes=self.cache_minutes)).strftime(
            "%Y-%m-%d %H:%M:%S"
        )
        sql = (
            f"select * from net_pdd_proxy_log where source = 'app' and mall_id = '{mall_id}' and create_time > '{cache_time}'"
            + " and base_url = 'https://api.pinduoduo.com/mall/{}/info'"
        )
        cache = self.to_db.find(sql, to_json=True)
        if cache:
            for i in cache:
                response = i["response"]
                if response and "error_code" not in response:
                    logger.warning(f"获取到店铺统计信息为: 使用缓存 {mall_id} {response}")
                    return response

        url = f"https://api.pinduoduo.com/mall/{mall_id}/info"
        resp = self.session.get(url).json()
        logger.info(f"获取到店铺统计信息为：{resp}")
        save_item = {
            "request_url": url,
            "base_url": "https://api.pinduoduo.com/mall/{}/info",
            "mall_id": mall_id,
            "page_no": None,
            "response": json.dumps(resp, ensure_ascii=False),
            "source": "app",
            "headers": self.session.headers,
        }
        self.to_db.add_smart("net_pdd_proxy_log", save_item)
        goods_num = resp.get("goods_num")
        mall_sales = resp.get("mall_sales")
        mall_desc = resp.get("mall_desc")
        mall_name = resp.get("mall_name")
        if mall_name:
            self.to_db.add_smart(
                "net_pdd_shop_info",
                {
                    "mall_id": mall_id,
                    "mall_goods_num": goods_num,
                    "mall_sales": mall_sales if mall_sales != "" else None,
                    "mall_desc": mall_desc,
                    "shop_name": mall_name,
                    "last_crawler_time": datetime.datetime.now(),
                },
                update_columns=[
                    "mall_goods_num",
                    "mall_sales",
                    "mall_desc",
                    "shop_name",
                    "last_crawler_time",
                ],  # update_columns
            )
        if resp.get("error_code"):
            if "verify_auth_token" in resp:
                verify = PddAppVerify(resp.get("verify_auth_token")).run()
                if verify:
                    self.session.headers.update({"VerifyAuthToken": resp.get("verify_auth_token")})
                    raise VerifyError(f"{resp.get('error_code')} {resp.get('error_message')}")
            raise Exception(MSG_MAP.get(resp.get("error_code")))
        else:
            time.sleep(random.randint(5, 10))
        return resp

    def get_mall_detail(self, mall_id, msn=""):
        """
        {"dsr": {"desc_score": 0, "desc_status": 0, "service_score": 0, "hide_rank_info": 1, "service_status": 0, "logistics_score": 0, "desc_rank_status": 0, "logistics_status": 0, "desc_rank_percent": 0, "is_show_mall_star": true, "service_rank_status": 0, "service_rank_percent": 0, "logistics_rank_status": 0, "logistics_rank_percent": 0}, "biz_ext": {"certificate_enhance": "false", "promise_info_enhance": "false"}, "bg_image_info": {"url": "https://funimg.pddpic.com/fa1ff096-b428-403a-8ff3-e247939ec8c4.png", "type": 3}, "mall_basic_info": {"is_open": 1, "mall_id": 100312875, "has_goods": false, "logo_list": [], "mall_desc": "该商户暂未上传店铺简介", "mall_logo": "http://t16img.yangkeduo.com/pdd_ims/img_check/v3/FFFFFF0000000820200112155026235/74d3f8ab272d4a7faeb134eb1061d672.png", "mall_name": "楼迪家居生活", "mall_authorize_logo": {"logo_list": [], "show_logo": false, "is_authorize": false}, "hit_mall_diversion_gray": true, "hit_mall_click_header_display_lego_layer_gray": false}, "mall_head_tag_vo": {"title": "店铺详情"}, "mall_label_volist": [], "mall_licence_info": {"show_icon": true, "is_certificated": true, "mall_publicity_document": "该店将于近期关闭，如有售后问题请及时处理"}, "guarantee_money_info": {"desc": "保证金"}, "mall_notification_info": {}, "mall_social_label_list": [], "mall_details_brand_list": [], "mall_details_label_list": [], "mall_authorize_licence_vo": {"desc": "已上传经营执照", "title": "店铺资质", "jump_url": "mall_certificates.html?mall_id=100312875"}}
        """
        cache_time = (datetime.datetime.now() - datetime.timedelta(minutes=self.cache_minutes)).strftime(
            "%Y-%m-%d %H:%M:%S"
        )
        sql = (
            f"select * from net_pdd_proxy_log where source = 'app' and mall_id = '{mall_id}' and create_time > '{cache_time}'"
            + " and base_url = 'https://api.pinduoduo.com/api/turing/mall/query_mall_detail_info'"
        )
        cache = self.to_db.find(sql, to_json=True)
        if cache:
            for i in cache:
                response = i["response"]
                if response and "error_code" not in response:
                    logger.warning(f"获取店铺详情为: 使用缓存 {mall_id} {response}")
                    return response

        url = "https://api.pinduoduo.com/api/turing/mall/query_mall_detail_info"
        params = {
            "mall_id": mall_id,
            "msn": msn,
            "source": "1",
            "contain_sold_goods": "true",
            "pdduid": self.pdd_user_id,
        }
        real_url = url + "?" + "&".join([f"{k}={v}" for k, v in params.items()])
        resp = self.session.get(url, params=params).json()
        logger.info(f"获取店铺详情为：{resp}")

        response = json.dumps(resp, ensure_ascii=False)
        save_item = {
            "request_url": real_url,
            "base_url": "https://api.pinduoduo.com/api/turing/mall/query_mall_detail_info",
            "mall_id": mall_id,
            "page_no": None,
            "response": response,
            "source": "app",
            "headers": self.session.headers,
        }
        self.to_db.add_smart("net_pdd_proxy_log", save_item)

        goods_num = resp.get("goods_num")
        mall_sales = resp.get("mall_sales")
        mall_desc = resp.get("mall_desc")
        mall_name = resp.get("mall_name")
        if mall_name:
            self.to_db.add_smart(
                "net_pdd_shop_info",
                {
                    "mall_id": mall_id,
                    "mall_goods_num": goods_num,
                    "mall_sales": mall_sales if mall_sales != "" else None,
                    "mall_desc": mall_desc,
                    "shop_name": mall_name,
                    "last_crawler_time": datetime.datetime.now(),
                },
                update_columns=[
                    "mall_goods_num",
                    "mall_sales",
                    "mall_desc",
                    "shop_name",
                    "last_crawler_time",
                ],  # update_columns
            )
        time.sleep(random.randint(5, 10))

    def get_goods_info(self, goods_id):
        """ """
        url = f"https://api.pinduoduo.com/api/oak/integration/render?pdduid={self.pdd_user_id}"
        data = {
            "address_list": [],
            "page_sn": "10014",
            "page_id": f"10014_{int(time.time() * 1000)}_0{''.join(random.choices(string.digits, k=9))}",
            "goods_id": goods_id,
            "client_time": int(time.time() * 1000),
            # "phone_model": "PHP110",
            "page_from": "39",
            "page_version": "7",
            "refer_page_sn": "10039",
            "refer_page_el_sn": "99755",
            "pic_w": 0,
            "pic_h": 0,
            "has_pic_url": 1,
            "_oc_trace_mark": "199",
            # "_oak_rcto": "YWIaqZ-8O30tzgY8am-v4MIjYPxrLzexvbzf9sz4gxd1-hE3aHvE40kC",
            "extend_map": {"_oc_trace_mark": "199"},
            "union_pay_installed": False,
            "client_lab": {"mall_h5_url_preload_enable": "1"},
            "is_sys_minor": 0,
            "cached_templates": [
                "4e176590c97383e90287ec1099255a89",
                "eb8d01a667687f88b7a621b730023e81",
                "f20e57f83229a4605587ad79351828b8",
                "7400c5a57ede335f73d573db43d39f3f",
                "4cb686aa9d52ed9214853abb540f1927",
                "b5bbbc60019576a86f3cfb8565846a73",
                "d0e12249136dbcf5ae9a95fdf64fda50",
                "b02e5cf12a800343e2470e8238d49790",
                "d719271f3c71c8b9a41efd2c2ce1ed03",
                "6cc85c7b0a90fc2a800018fd1f89f670",
                "59bc6ac22b50322d3a870ccc880d96b1",
                "8c6858b96a325699cc23db4f6310032b",
                "f030ffc32e219d7e5c19aed6079e7eb2",
            ],
            "impr_tips": [],
        }
        data = json.dumps(data, separators=(",", ":"))
        response = self.session.post(url, data=data)
        print(response.text)

    def run_mall(self, mall_id, msn=""):
        mall_total_info = self.get_mall_total(mall_id=mall_id)

        self.get_mall_page_info(mall_id=mall_id, msn=msn)
        self.get_mall_detail(mall_id=mall_id, msn=msn)

        goods_num = mall_total_info.get("goods_num")
        total_page_num = math.ceil(goods_num / 20)
        if total_page_num >= 10:
            total_page_num = 10  # 最多爬9页
        for page_no in range(1, total_page_num + 1):
            logger.info(f"获取第 {page_no} / {total_page_num} 页商品")
            self.get_mall_goods_info(mall_id=mall_id, msn=msn, page_no=page_no)

        SyncPddSource().run_by_mall_ids([mall_id])

    def auto_tun(self, user=None):
        try:
            count = self.redis_db.sget_count("pdd_app_wfq_mall_id_210")
            if count > 100:
                self.auto_rerun()
            else:
                datas = self.to_db.find(
                    "select mall_id, msn from net_pdd_shop_info_task where channel = 'wfq_test_210' limit 1000",
                    to_json=True,
                )
                self.redis_db.sadd("pdd_app_wfq_mall_id_210", values=[json.dumps(i) for i in datas])
                self.auto_rerun()
                time.sleep(60)
        except Exception as e:
            logger.error(e)
            if "登录已过期" in str(e):
                user.delete()

    def auto_rerun(self):
        mall_dict_ids = self.redis_db.sget("pdd_app_wfq_mall_id_210", count=100, is_pop=False)
        mall_dict_ids = random.choices(mall_dict_ids, k=5)
        for mall_dict_str in mall_dict_ids:
            mall_dict = json.loads(mall_dict_str)
            self.run_mall(mall_id=mall_dict["mall_id"], msn=mall_dict["msn"] or "")
            self.redis_db.srem("pdd_app_wfq_mall_id_210", mall_dict_str)

    def get_cert(self, mall_id):
        url = "https://api.pinduoduo.com/api/turing/mall/query_mall_licence_certificate"
        params = {
            "pdduid": self.pdd_user_id,
            "mall_id": mall_id,
            "msn": "",
        }
        headers = {
            **self.session.headers,
            "anti-token": self.gen_anti_content(device_id=self.session.headers["etag"]),
            "Referer": f"https://m.pinduoduo.net/mall_certificates.html?mall_id={mall_id}",
        }
        print(headers)
        params.update(
            {
                "anti_content": gen_anti_content(),
                "antiContent": gen_anti_content(),
            }
        )
        real_url = url + "?" + "&".join([f"{k}={v}" for k, v in params.items()])
        resp = self.session.get(real_url, headers=headers).json()
        logger.info(f"获取证书详情为：{resp}")
        if resp.get("verify_auth_token"):
            verify_auth_token = resp.get("verify_auth_token")
            PddSliderVerify(verify_auth_token).run()
            headers = {
                **self.session.headers,
                "anti-token": self.gen_anti_content(device_id=self.session.headers["etag"]),
                "Referer": f"https://m.pinduoduo.net/mall_certificates.html?mall_id={mall_id}",
                "VerifyAuthToken": verify_auth_token,
            }
            print(headers)
            params.update(
                {
                    "anti_content": gen_anti_content(),
                    "antiContent": gen_anti_content(),
                }
            )
            real_url = url + "?" + "&".join([f"{k}={v}" for k, v in params.items()])
            resp = self.session.get(real_url, headers=headers).json()
            logger.info(f"获取证书验证后详情为：{resp}")


if __name__ == "__main__":
    tt = PDDAppUserPool(redis_key="user:pdd:app", max_users_num=1, keep_alive=True)
    user = tt.get_user_by_id("19059631211")
    user.refresh()
    print(user)
    # test = AppCrawler(cookie=user.cookies, headers=user.headers, cache_minutes=60 * 24)
    # test.run_mall('1107330')
    # test.get_cert('1104948')
